{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assess the performance of a LoRAHub model on RE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-10T05:39:15.719117Z",
     "iopub.status.busy": "2024-05-10T05:39:15.718802Z",
     "iopub.status.idle": "2024-05-10T05:39:22.550183Z",
     "shell.execute_reply": "2024-05-10T05:39:22.549635Z",
     "shell.execute_reply.started": "2024-05-10T05:39:15.719098Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2024-05-10 13:39:17.706498: I tensorflow/core/util/port.cc:111] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-05-10 13:39:17.708894: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-05-10 13:39:17.740103: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-05-10 13:39:17.740130: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-05-10 13:39:17.740149: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-05-10 13:39:17.746020: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-05-10 13:39:17.746811: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-05-10 13:39:18.553439: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "from lorahub.algorithm import *\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-05-10T05:39:22.551909Z",
     "iopub.status.busy": "2024-05-10T05:39:22.551088Z",
     "iopub.status.idle": "2024-05-10T05:39:22.556990Z",
     "shell.execute_reply": "2024-05-10T05:39:22.556482Z",
     "shell.execute_reply.started": "2024-05-10T05:39:22.551886Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_examples_for_learning(p):\n",
    "    res = []\n",
    "    for path in p:\n",
    "        with open(path, 'r') as f:\n",
    "              js = json.load(f)\n",
    "        res.extend(js)\n",
    "    return res\n",
    "\n",
    "\n",
    "def get_examples_for_inference(p):\n",
    "    res = []\n",
    "    for path in p:\n",
    "        with open(path, 'r') as f:\n",
    "              js = json.load(f)\n",
    "        res.extend(js)\n",
    "    return res\n",
    "\n",
    "\n",
    "def get_lora_module_list(task):\n",
    "    pefix = \"/mnt/workspace/t5_chat/save/\"\n",
    "\n",
    "    all_loras = ['re_task/duie_re', 'ee_task/ace05_tuple_ee', 'ee_task/casie_tuple_ee', 'ee_task/duee_tuple_ee', 'ee_task/genia_tuple_ee',\n",
    "                 'ee_task/phee_tuple_ee', 'ner_task/ace_ner', 'ner_task/cnerta_ner', 'ner_task/conll_ner',\n",
    "                 'ner_task/multinerd_ner', 're_task/conll04_re', 're_task/gids_re',\n",
    "                 're_task/nyt11_re']\n",
    "\n",
    "    ner_loras = [\"ner_task/cnerta_ner\", \"ner_task/ace_ner\", \"ner_task/conll_ner\", \"ner_task/multinerd_ner\"]\n",
    "\n",
    "    re_loras = ['re_task/duie_re', 're_task/conll04_re',  're_task/gids_re', 're_task/nyt11_re']\n",
    "\n",
    "    ee_loras = ['ee_task/duee_tuple_ee', 'ee_task/ace05_tuple_ee', 'ee_task/casie_tuple_ee', 'ee_task/genia_tuple_ee',\n",
    "                'ee_task/phee_tuple_ee']\n",
    "\n",
    "    if task == 'ner':\n",
    "        p = ner_loras\n",
    "    elif task == 're':\n",
    "        p = re_loras\n",
    "    elif task == 'ee':\n",
    "        p = ee_loras\n",
    "    elif task == 'all':\n",
    "        p = all_loras\n",
    "    else:\n",
    "        raise ValueError(\"task must be one of 'ner', 're', 'ee', 'all'\")\n",
    "\n",
    "    return [pefix + x for x in p]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-05-10T05:39:22.558195Z",
     "iopub.status.busy": "2024-05-10T05:39:22.557755Z",
     "iopub.status.idle": "2024-05-10T09:39:46.568022Z",
     "shell.execute_reply": "2024-05-10T09:39:46.567258Z",
     "shell.execute_reply.started": "2024-05-10T05:39:22.558177Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "modules: ['/mnt/workspace/t5_chat/save/re_task/duie_re', '/mnt/workspace/t5_chat/save/ee_task/ace05_tuple_ee', '/mnt/workspace/t5_chat/save/ee_task/casie_tuple_ee', '/mnt/workspace/t5_chat/save/ee_task/duee_tuple_ee', '/mnt/workspace/t5_chat/save/ee_task/genia_tuple_ee', '/mnt/workspace/t5_chat/save/ee_task/phee_tuple_ee', '/mnt/workspace/t5_chat/save/ner_task/ace_ner', '/mnt/workspace/t5_chat/save/ner_task/cnerta_ner', '/mnt/workspace/t5_chat/save/ner_task/conll_ner', '/mnt/workspace/t5_chat/save/ner_task/multinerd_ner', '/mnt/workspace/t5_chat/save/re_task/conll04_re', '/mnt/workspace/t5_chat/save/re_task/gids_re', '/mnt/workspace/t5_chat/save/re_task/nyt11_re']\n",
      "> Begin to load lora modules\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:154: UserWarning: Could not find a config file in /mnt/workspace/t5_chat/model_save/ - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "2it [00:00, 12.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Loading /mnt/workspace/t5_chat/save/re_task/duie_re ...\n",
      "> Loading /mnt/workspace/t5_chat/save/ee_task/ace05_tuple_ee ...\n",
      "> Loading /mnt/workspace/t5_chat/save/ee_task/casie_tuple_ee ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4it [00:00, 12.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Loading /mnt/workspace/t5_chat/save/ee_task/duee_tuple_ee ...\n",
      "> Loading /mnt/workspace/t5_chat/save/ee_task/genia_tuple_ee ...\n",
      "> Loading /mnt/workspace/t5_chat/save/ee_task/phee_tuple_ee ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 12.73it/s]/opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:154: UserWarning: Could not find a config file in /mnt/workspace/ChatLM-mini-Chinese/model_save/ - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n",
      "8it [00:00, 12.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Loading /mnt/workspace/t5_chat/save/ner_task/ace_ner ...\n",
      "> Loading /mnt/workspace/t5_chat/save/ner_task/cnerta_ner ...\n",
      "> Loading /mnt/workspace/t5_chat/save/ner_task/conll_ner ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10it [00:00, 12.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Loading /mnt/workspace/t5_chat/save/ner_task/multinerd_ner ...\n",
      "> Loading /mnt/workspace/t5_chat/save/re_task/conll04_re ...\n",
      "> Loading /mnt/workspace/t5_chat/save/re_task/gids_re ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13it [00:01, 12.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Loading /mnt/workspace/t5_chat/save/re_task/nyt11_re ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Running tokenizer on dataset: 100%|██████████| 20674/20674 [00:07<00:00, 2935.10 examples/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['input_ids', 'attention_mask', 'labels'],\n",
      "    num_rows: 20674\n",
      "})\n",
      "> Begin to perform gradient-free optimization ...\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008147196782098262\n",
      "39 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.018792253700169084\n",
      "38 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010048381173436195\n",
      "37 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010123453987545062\n",
      "36 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010369840549234488\n",
      "35 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010198437864627614\n",
      "34 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010094522883594864\n",
      "33 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010112326895356504\n",
      "32 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010118578319529833\n",
      "31 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010106399851457577\n",
      "30 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010471147940939687\n",
      "29 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010052871201648526\n",
      "28 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010163093699916845\n",
      "27 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010612456697654794\n",
      "26 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.023949809880356367\n",
      "25 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.010770521665829258\n",
      "24 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.009546764347282691\n",
      "23 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.00848434782330762\n",
      "22 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.00904752470183743\n",
      "21 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008474581239724381\n",
      "20 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.00906842104602417\n",
      "19 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008489377406688437\n",
      "18 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.009048446431997691\n",
      "17 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008491420786984998\n",
      "16 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.00904265621270546\n",
      "15 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.00849384262790559\n",
      "14 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.009088661535627578\n",
      "13 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008482762292457553\n",
      "12 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.009091361946760189\n",
      "11 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.00863162604538956\n",
      "10 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008382019518694672\n",
      "9 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:57,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008264985719755959\n",
      "8 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008445887604108394\n",
      "7 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008229857902818476\n",
      "6 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008481863953328945\n",
      "5 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.00826923646105214\n",
      "4 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008469339799380696\n",
      "3 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008248852270112614\n",
      "2 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.00820585208170843\n",
      "1 remaining budget and 0 running jobs\n",
      "Launching 1 jobs with new suggestions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [05:58,  1.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating fitness with value 0.008287695982098414\n",
      "0 remaining budget and 0 running jobs\n",
      "module_weights: [1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "modules = get_lora_module_list(\"all\")\n",
    "print(\"modules:\", modules)\n",
    "\n",
    "ds_train_path = [\"/mnt/workspace/ds/re/duie_dev.json\"]\n",
    "ds_test_path = [\"/mnt/workspace/ds/re/duie_dev.json\"]\n",
    "\n",
    "# construct input list and output list\n",
    "example_inputs, examples_outputs = [], []\n",
    "for example in get_examples_for_learning(ds_train_path):\n",
    "    example_inputs.append(example[\"prompt\"])\n",
    "    examples_outputs.append(example[\"response\"])\n",
    "\n",
    "# perform LoRAHub learning\n",
    "module_weights, model, tokenizer = lorahub_learning(lora_module_list=modules,\n",
    "                                                    example_inputs=example_inputs,\n",
    "                                                    example_outputs=examples_outputs,\n",
    "                                                    max_inference_step=40,\n",
    "                                                    batch_size=32,\n",
    "                                                   )\n",
    "\n",
    "print(\"module_weights:\", module_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2024-05-10T09:41:12.565151Z",
     "iopub.status.busy": "2024-05-10T09:41:12.564829Z",
     "iopub.status.idle": "2024-05-10T09:41:13.457563Z",
     "shell.execute_reply": "2024-05-10T09:41:13.456996Z",
     "shell.execute_reply.started": "2024-05-10T09:41:12.565132Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "save_path = \"/mnt/workspace/save/comb_re\"\n",
    "model.save_pretrained(save_path)\n",
    "tokenizer.save_pretrained(save_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## performance of lorahub model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "example_inputs, examples_outputs = [], []\n",
    "with open(\"/mnt/workspace/ds/re/duie_dev.json\", \"r\", encoding=\"utf-8\") as f:\n",
    "        js = json.load(f)\n",
    "for example in js:\n",
    "    example_inputs.append(example[\"prompt\"])\n",
    "    examples_outputs.append(example[\"response\"])\n",
    "\n",
    "example_predictions1, perf = assess_task_performance(example_inputs=example_inputs,\n",
    "                                                    model_or_name_path=model,\n",
    "                                                    tokenizer_or_tokenizer_path=tokenizer,\n",
    "                                                    batch_size=32,\n",
    "                                                    # can set as None if you do not have the ground truth\n",
    "                                                    example_outputs=examples_outputs,\n",
    "                                                    task = \"re\"\n",
    "                                                    )\n",
    "# print(\"example_predictions:\", example_predictions)\n",
    "# print(\"task accuracy:\", perf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-05-10T10:39:12.390135Z",
     "iopub.status.busy": "2024-05-10T10:39:12.389803Z",
     "iopub.status.idle": "2024-05-10T10:39:12.461988Z",
     "shell.execute_reply": "2024-05-10T10:39:12.461445Z",
     "shell.execute_reply.started": "2024-05-10T10:39:12.390115Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "js = []\n",
    "save_op_path = \"/mnt/workspace/save/outcome.json\"\n",
    "for x, y in zip(example_predictions1, examples_outputs):\n",
    "    js.append({\"prediction\":x, \"acrual\":y})\n",
    "with open(save_op_path, \"w\", encoding= \"utf-8\") as f:\n",
    "    json.dump(js, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-05-10T10:34:11.928771Z",
     "iopub.status.busy": "2024-05-10T10:34:11.928431Z",
     "iopub.status.idle": "2024-05-10T10:34:12.091720Z",
     "shell.execute_reply": "2024-05-10T10:34:12.091200Z",
     "shell.execute_reply.started": "2024-05-10T10:34:11.928752Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'precision': 0.6341140858729832, 'recall': 0.570789905296016, 'f1': 0.6007879825697142}\n"
     ]
    }
   ],
   "source": [
    "def ee_calculate_metrics(predicted: List[str], actual: List[str]):\n",
    "    def ee_parse_tuple_string(tuple_string: str) -> List[Tuple[str, str, str]]:\n",
    "        tuple_string = tuple_string.replace('，',',').replace('（','(').replace('）', ')').replace(\"：\", \":\").replace(\" \", \"\")\n",
    "        pattern = re.compile(r'\\(([^,]*),? ?([^,]*)?,? ?([^)]*)?\\)')\n",
    "        tuples = pattern.findall(tuple_string)\n",
    "        tuples = set(('EMPTY', 'EMPTY', 'EMPTY') if t == ('', '', '') else t for t in tuples)\n",
    "        return tuples\n",
    "\n",
    "    tp = 0\n",
    "    fp = 0\n",
    "    fn = 0\n",
    "\n",
    "    for p, a in zip(predicted, actual):\n",
    "        predicted_tuples = ee_parse_tuple_string(p)\n",
    "        actual_tuples = ee_parse_tuple_string(a)\n",
    "\n",
    "        tp_temp = 0\n",
    "        for pt in predicted_tuples:\n",
    "            for at in actual_tuples:\n",
    "                if (pt[0] in at[0] and pt[1] in at[1] and pt[2] in at[2]) or (at[0] in pt[0] and at[1] in pt[1] and at[2] in pt[2]):\n",
    "                    tp_temp += 1\n",
    "                    break\n",
    "\n",
    "        tp += tp_temp\n",
    "        fp += len(predicted_tuples) - tp_temp\n",
    "        fn += len(actual_tuples) - tp_temp\n",
    "\n",
    "    precision = tp / (tp + fp) if (tp + fp) > 0 else 1\n",
    "    recall = tp / (tp + fn) if (tp + fn) > 0 else 1\n",
    "    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 1\n",
    "    \n",
    "    # print(tp, fp, fn)\n",
    "\n",
    "    return {\"precision\": precision, \"recall\": recall, \"f1\": f1}\n",
    "print(ee_calculate_metrics(example_predictions1, examples_outputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-05-10T10:49:00.756085Z",
     "iopub.status.busy": "2024-05-10T10:49:00.755787Z",
     "iopub.status.idle": "2024-05-10T10:49:13.829738Z",
     "shell.execute_reply": "2024-05-10T10:49:13.829206Z",
     "shell.execute_reply.started": "2024-05-10T10:49:00.756067Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-05-10 18:49:00,778 - modelscope - INFO - PyTorch version 2.1.2+cu121 Found.\n",
      "2024-05-10 18:49:00,780 - modelscope - INFO - TensorFlow version 2.14.0 Found.\n",
      "2024-05-10 18:49:00,781 - modelscope - INFO - Loading ast index from /mnt/workspace/.cache/modelscope/ast_indexer\n",
      "2024-05-10 18:49:00,809 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 9624771835d15245f3715ef006c0d0fa and a total number of 976 components indexed\n",
      "2024-05-10 18:49:05,564 - modelscope - INFO - [master 2f34269] 'upload model'\n",
      " 8 files changed, 52691 insertions(+), 52 deletions(-)\n",
      " delete mode 100644 README.md\n",
      " create mode 100644 config.json\n",
      " create mode 100644 generation_config.json\n",
      " create mode 100644 model.safetensors\n",
      " create mode 100644 special_tokens_map.json\n",
      " create mode 100644 tokenizer.json\n",
      " create mode 100644 tokenizer_config.json\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from modelscope.hub.api import HubApi\n",
    "\n",
    "YOUR_ACCESS_TOKEN = 'b2b49ccc-e06b-40b5-8869-5069e7436fd2'\n",
    "# 请注意ModelScope平台针对SDK访问和git访问两种模式，提供两种不同的访问令牌(token)。此处请使用SDK访问令牌。\n",
    "\n",
    "\n",
    "api = HubApi()\n",
    "api.login(YOUR_ACCESS_TOKEN)\n",
    "api.push_model(\n",
    "    model_id=\"Wojtek/t5-RE-CN\", \n",
    "    model_dir=\"/mnt/workspace/save/comb_re\" # 本地模型目录，要求目录中必须包含configuration.json\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## performance of only lora model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-05-08T14:08:51.627863Z",
     "iopub.status.busy": "2024-05-08T14:08:51.627624Z",
     "iopub.status.idle": "2024-05-08T14:35:48.964893Z",
     "shell.execute_reply": "2024-05-08T14:35:48.964399Z",
     "shell.execute_reply.started": "2024-05-08T14:08:51.627841Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2024-05-08 22:08:53.820564: I tensorflow/core/util/port.cc:111] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-05-08 22:08:53.822892: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-05-08 22:08:53.853265: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-05-08 22:08:53.853288: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-05-08 22:08:53.853306: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-05-08 22:08:53.859001: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-05-08 22:08:53.859575: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-05-08 22:08:54.509141: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "Running tokenizer on dataset: 100%|██████████| 20674/20674 [00:06<00:00, 3016.40 examples/s]\n",
      "/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2663: UserWarning: `max_length` is ignored when `padding`=`True` and there is no truncation strategy. To pad to max length, use `padding='max_length'`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "task accuracy: {'precision': 0.5539419736471471, 'recall': 0.531741024485504, 'f1': 0.5426145073272427}\n"
     ]
    }
   ],
   "source": [
    "from lorahub.algorithm import *\n",
    "import json\n",
    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
    "import peft\n",
    "\n",
    "example_inputs, examples_outputs = [], []\n",
    "with open(\"/mnt/workspace/ds/re/duie_dev.json\", \"r\", encoding=\"utf-8\") as f:\n",
    "        js = json.load(f)\n",
    "for example in js:\n",
    "    example_inputs.append(example[\"prompt\"])\n",
    "    examples_outputs.append(example[\"response\"])\n",
    "\n",
    "lora_path = \"/mnt/workspace/t5_chat/save/re_task/duie_re\"\n",
    "model_path = \"/mnt/workspace/t5_chat/model_save\"\n",
    "device = \"cuda\"\n",
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(model_path).to(\"cuda\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(lora_path)\n",
    "\n",
    "model.load_adapter(lora_path,adapter_name=\"duie\")\n",
    "model.set_adapter(\"duie\")\n",
    "\n",
    "example_predictions2, perf = assess_task_performance(example_inputs=example_inputs,\n",
    "                                                    model_or_name_path=model,\n",
    "                                                    tokenizer_or_tokenizer_path=tokenizer,\n",
    "                                                    batch_size=32,\n",
    "                                                    # can set as None if you do not have the ground truth\n",
    "                                                    example_outputs=examples_outputs,\n",
    "                                                    task = \"re\"\n",
    "                                                    )\n",
    "# print(\"example_predictions:\", example_predictions)\n",
    "print(\"task accuracy:\", perf)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
